from time import sleep

import pandas as pd
import requests
from lxml import etree

urls = ['https://heyuan.58.com/xinfang/']

ershou_urls = ['https://heyuan.58.com/ershoufang/']

headers = {
    'authority': 'heyuan.58.com',
    'method': 'GET',
    'path': '/xinfang/',
    'scheme': 'https',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Cache-Control': 'max-age=0',
    'Cookie': 'f=n; isp=true; 58tw_ctid=891; aQQ_ajkguid=0CC392F1-622A-1414-9C1A-SX1117080325; sessid=400899E0-D21E-5820-D88E-SX1117080325; from_source=58tw; id58=CrIgq2VWrc25fvQ+FL7YAg==; 58tj_uuid=9d021b54-94b1-4e4b-a55f-cb6c3c20e957; new_uv=1; qz_gdt=; als=0; new_session=0; myLat=""; myLon=""; mcity=kaipingshi; utm_source=market; spm=u-2d2yxv86y3v43nkddh1.BDPCPZ_BT; init_refer=https%253A%252F%252Fwww.baidu.com%252Fother.php%253Fsc.000000jPDQBGTF0l5ZEaEg2T8xWXVFmn_HyNR9IN4z1XZzpCatfs7vUJgTct2YSyVtNTUZvfjEjZWZ_BYyUzMuu9Yj0y3MOlBpacgOtBbZ2W07DfXWspYZwZECV2tW3ztxWnd4RF8kBrgSHi506Ho69_fUPdbpt8GSDTDztzhnhxJ1xEmokOMFKk1ulXqbhB1xOAhoe4k_8nOOWbaw2Cw9_jqo6_.DY_NR2Ar5Od66z3PrrW6ButVvkDj3n-vHwYxw_vU85YIMAQV8qhORGyAp7WIu8L6.TLFWgv-b5HDkrfK1ThPGujYknHb0THY0IAYqPHWPoQ5Z0ZN1ugFxIZ-suHYs0A7bgLw4TARqnsKLULFb5HR31pz1ksKzmLmqn0KdThkxpyfqnHRdn1T1nH6vnsKVINqGujYkPHTsPH63rfKVgv-b5HDzrjn3rHc30AdYTAkxpyfqnHc3nWm0TZuxpyfqn0KGuAnqiD4a0ZKGujYY0APGujY3n6KWThnqrjRz%2526ck%253D6588.2.136.331.163.275.257.356%2526; wmda_uuid=0ef677fdccfc91b11575d79c47f2b623; wmda_new_uuid=1; wmda_session_id_2385390625025=1700179878193-92ca866f-d94a-98d4; wmda_visited_projects=%3B2385390625025; is_58_pc=1; xxzlclientid=5a278a94-e1e2-428a-8db5-1700179894270; xxzlxxid=pfmx532AcI3g8RcoT0WboV3GAaSa8ZMx9YXqMvjLrj+Gb9Ev74b8ABpC7WIaer/QpVas; f=n; city=heyuan; 58home=heyuan; commontopbar_new_city_info=10467%7C%E6%B2%B3%E6%BA%90%7Cheyuan; commontopbar_ipcity=kaipingshi%7C%E5%BC%80%E5%B9%B3%7C0; 58_ctid=10467; commontopbar_new_city_info=226%7C%E6%B2%B3%E6%BA%90%7Che; wmda_session_id_8788302075828=1700180118276-11a01035-9f17-b9fd; wmda_visited_projects=%3B2385390625025%3B8788302075828; ajk-appVersion=; fzq_h=055987dc24ef64f2792e10677dd08c35_1700180141547_bdb3392a44ff43b69ca7efc6c57a747d_1946559671; xxzl_cid=70ab307df89f4833ae43b5b33986a7cd; xxzl_deviceid=mRpvaX73VjpY4SyZY0OLMBHlWirIvvwMJ6iWO2+iBMTB7CILhwbDNRXlZ/yHaBB2; ctid=226; xxzl_cid=70ab307df89f4833ae43b5b33986a7cd; xxzl_deviceid=mRpvaX73VjpY4SyZY0OLMBHlWirIvvwMJ6iWO2+iBMTB7CILhwbDNRXlZ/yHaBB2; xxzlbbid=pfmbM3wxMDI5M3wxLjMuMHwxNzAwMTgwMTU2MTQyfHl3TjB2c0o4aFE5YkxPaHZsSHZJckMwdUN4TU1nUzlEOWcvUkRPczZUQUE9fDQ1MTJmNThmZjg0MjE5MGQ0ZDQ2ODgzOGQ3MWNkNmI3XzE3MDAxODAxMTY5ODlfOTdkNjllYTYzZWVkNDVhOWE2M2RhZTMwN2RiZTU1NDdfMTk0NjU1OTY3MXw1ODE2NzM3YzlkOWFhYzY2YTk0NWU4ZTUyMzkwMGVkYl8xNzAwMTgwMTU1ODA0XzEzOA==',
    'If-Modified-Since': 'Fri, 17 Nov 2023 00:15:53 GMT',
    'Referer': 'https://heyuan.58.com/ershoufang/?PGTID=0d3091a8-028e-34a9-1b5f-47120a57934d&ClickID=1',
    'Sec-Ch-Ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Microsoft Edge";v="116"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.69'
}

ershou_headers = {
    'authority': 'heyuan.58.com',
    'method': 'GET',
    'path': '/ershoufang/',
    'scheme': 'https',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Cache-Control': 'max-age=0',
    'Cookie': 'f=n; commontopbar_new_city_info=10467%7C%E6%B2%B3%E6%BA%90%7Cheyuan; userid360_xml=9AAEB36D31C801BA62A60092A445972C; time_create=1702772145287; 58tw_ctid=891; aQQ_ajkguid=0CC392F1-622A-1414-9C1A-SX1117080325; sessid=400899E0-D21E-5820-D88E-SX1117080325; from_source=58tw; id58=CrIgq2VWrc25fvQ+FL7YAg==; 58tj_uuid=9d021b54-94b1-4e4b-a55f-cb6c3c20e957; als=0; myLat=""; myLon=""; mcity=kaipingshi; wmda_uuid=0ef677fdccfc91b11575d79c47f2b623; wmda_new_uuid=1; wmda_visited_projects=%3B2385390625025; is_58_pc=1; xxzlclientid=5a278a94-e1e2-428a-8db5-1700179894270; xxzlxxid=pfmx532AcI3g8RcoT0WboV3GAaSa8ZMx9YXqMvjLrj+Gb9Ev74b8ABpC7WIaer/QpVas; f=n; city=heyuan; 58home=heyuan; commontopbar_new_city_info=10467%7C%E6%B2%B3%E6%BA%90%7Cheyuan; commontopbar_ipcity=kaipingshi%7C%E5%BC%80%E5%B9%B3%7C0; 58_ctid=10467; commontopbar_new_city_info=226%7C%E6%B2%B3%E6%BA%90%7Che; wmda_visited_projects=%3B2385390625025%3B8788302075828; ajk-appVersion=; fzq_h=055987dc24ef64f2792e10677dd08c35_1700180141547_bdb3392a44ff43b69ca7efc6c57a747d_1946559671; lp_lt_ut=21c25a7fecf9389d1ee73c0d725a61f7; ved_loupans=482150%3A488053; xxzl_cid=70ab307df89f4833ae43b5b33986a7cd; xxzl_deviceid=+OufKDwLUGqH4P0+AzGhAtzYBJC7fJaFp9Zqi6RUJfnu/Bhb0r0i2tNVwehl9XsJ; xxzlbbid=pfmbM3wxMDI5M3wxLjMuMXwxNzAwMTkwMzQwNzMzfHJFVmFZWDRJTU5nQ2tuNW1LZDVtR1dDa0NjQVJXcUVkandLSmpFRzhESVk9fGY0YzY4NGZmNjRhMjhmZWIxNmU1ZTc0YjRjNDJmZTQyXzE3MDAxOTAzMzg5NDdfMDgyM2FjYzRjYTgzNDMyNWJjNWE3YzgwN2Y2NjkwNjdfMTk0NjU1OTY3MXw0ZjhhY2ZlYzE3NzFmZjdjMzRjZjQ2NmY2NWViOGY5MV8xNzAwMTkwMzM5MzMwXzI1Ng==; wmda_session_id_2385390625025=1700203670185-70889970-804b-1a67; xxzlcid=70ab307df89f4833ae43b5b33986a7cd; xxzl-cid=70ab307df89f4833ae43b5b33986a7cd; new_uv=4; utm_source=; spm=; init_refer=https%253A%252F%252Fheyuan.58.com%252Fhouse.shtml%253FPGTID%253D0d100000-028e-30b7-f443-65813221915d%2526ClickID%253D6; ctid=10467; new_session=0; xxzl_cid=70ab307df89f4833ae43b5b33986a7cd; xxzl_deviceid=+OufKDwLUGqH4P0+AzGhAtzYBJC7fJaFp9Zqi6RUJfnu/Bhb0r0i2tNVwehl9XsJ',
    'If-Modified-Since': 'Fri, 17 Nov 2023 00:15:53 GMT',
    'Referer': 'https://heyuan.58.com/ershoufang/?PGTID=0d3091a8-028e-34a9-1b5f-47120a57934d&ClickID=1',
    'Sec-Ch-Ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Microsoft Edge";v="116"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.69'
}

div = []
ershou_div = []


def splice_url():
    urls.extend(['https://heyuan.58.com/xinfang/loupan/all/p{}/'.format(i) for i in range(2, 5)])
    ershou_urls.extend(['https://heyuan.58.com/ershoufang/p{}/'.format(i) for i in range(2, 51)])


def req_data():
    for url in urls:
        response = requests.get(url, headers=headers)
        html = response.content.decode()
        html_doc = etree.HTML(html)
        div.extend(html_doc.xpath('/html/body/div[3]/div[2]/div/div[2]/div'))
    data = handle_data()
    save_data_to_csv(data)


def req_ershou_data():
    for url in ershou_urls:
        response = requests.get(url, headers=ershou_headers)
        html = response.content.decode()
        html_doc = etree.HTML(html)
        ershou_div.extend(
            html_doc.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]/div/a/div[2]'))
        sleep(0.5)
    save_ershou_data_to_csv(handle_ershou_data())


def handle_ershou_data():
    ershou_div_div = [i.xpath('./div[1]/section/div|./div[2]') for i in ershou_div]
    ershou_data = []
    for d in ershou_div_div:
        name = ''.join(d[1].xpath('./p[1]/text()'))
        address = ','.join(d[1].xpath('./p[2]/span/text()'))
        huxing = ''.join(d[0].xpath('./p[1]/span/text()')) + '|' + ''.join(
            d[0].xpath('./p[2]/text()')) + '|' + ''.join(d[0].xpath('./p[3]/text()'))
        huxing = huxing.replace('\n', '').replace(' ', '')
        if len(d) == 4:
            tag = ','.join(d[2].xpath('./span/text()'))
            price = ''.join(d[3].xpath('./p[1]/span/text()')) + ',' + ''.join(d[3].xpath('./p[2]/text()'))
            ershou_data.append([name, address, huxing, tag, price])
        else:
            tag = ''
            price = ''.join(d[2].xpath('./p[1]/span/text()')) + ',' + ''.join(d[2].xpath('./p[2]/text()'))
            ershou_data.append([name, address, huxing, tag, price])
    return ershou_data


def handle_data():
    a = [i.xpath('./div/a|./a[2]') for i in div]
    span = [[','.join(j.xpath('.//span/text()')).replace(' ', '') for j in i] for i in a]
    return span


def save_data_to_csv(data):
    df = pd.DataFrame(data, columns=["name", "address", "huxing", "tag", "price"])
    df.to_csv('../static/data/data.csv', header=True, index=None)


def save_ershou_data_to_csv(data):
    df = pd.DataFrame(data, columns=["name", "address", "huxing", "tag", "price"])
    df.to_csv('../static/data/ershou_data.csv', header=True, index=None)


if __name__ == '__main__':
    splice_url()
    # req_data()
    req_ershou_data()
